{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "507f9ab1",
   "metadata": {},
   "source": [
    "### MDP过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "8c380ce2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:19:07.970518Z",
     "start_time": "2024-05-09T07:19:07.966529Z"
    }
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "04997bac",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:20:23.988732Z",
     "start_time": "2024-05-09T07:20:23.980726Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_next_State_value(discount_factor,rewards,old_state_values,transition_probabilities):\n",
    "    #用于储存计算得到下一阶段的状态值函数\n",
    "    new_state_values=[]\n",
    "    #使用for循环遍历old_state_value的列表索引,current_state_value表示当前状态的索引\n",
    "    for current_state_value in range(len(old_state_values)):\n",
    "        #检查当前元素是否为最后一个元素\n",
    "        if current_state_value!=len(old_state_values)-1:\n",
    "                #创建状态值函数的加权和\n",
    "                next_state_value_sum=0.0\n",
    "                #transition表示状态转移函数\n",
    "                for transition in transition_probabilities[current_state_value]:\n",
    "                    #从transition元素中获取转移概率和下一状态的索引，并将其加权值函数累加\n",
    "                    transition_probability=transition[0]\n",
    "                    next_state_index=transition[1]\n",
    "                    next_state_value_sum+=transition_probability*old_state_values[next_state_index]\n",
    "                #根据当前状态的即时奖励和折时因子,计算当前状态在下一阶段的预期收益,并将其添加到new_state_value\n",
    "                new_state_value=rewards[current_state_value]+discount_factor*next_state_value_sum\n",
    "                new_state_values.append(new_state_value)\n",
    "    new_state_values.append(0.0)\n",
    "    return new_state_values\n",
    "def calc_next_State_value_2(pi,lambd,transition_probabilities,rewards,identity_matrix):\n",
    "    p_mat=np.matrix(transition_probabilities)\n",
    "    r_mat=np.matrix(rewards)\n",
    "    i_mat=np.matrix(identity_matrix)\n",
    "    v_mat=(i_mat-lambd*p_mat).T*r_mat\n",
    "    return v_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "109034ef",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:15:05.666655Z",
     "start_time": "2024-05-09T07:15:05.661159Z"
    }
   },
   "outputs": [],
   "source": [
    "#配置参数\n",
    "discount_factor=1\n",
    "pi=0.5\n",
    "lambd=1\n",
    "rewards=[-2,-2,-2,10,1,-1,0]\n",
    "old_state_values=[0,0,0,0,0,0,0]\n",
    "'''\n",
    "定义了一个状态之间的转移概率\n",
    "每个子序列表示状态后续状态以及对应后续转移概率\n",
    "exp:[0.5,1]表示以0.5的概率转移到Class1\n",
    "'''\n",
    "transition_probabilities=[\n",
    "    [[0.5,1],[0.5,5]],\n",
    "    [[0.8,2],[0.2,6]],\n",
    "    [[0.6,3],[0.4,4]],\n",
    "    [[1,6]],\n",
    "    [[0.2,0],[0.4,1],[0.4,2]],\n",
    "    [[0.1,0],[0.9,5]],\n",
    "    [[0,0]]\n",
    "]\n",
    "new_state_values=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1c009419",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:15:06.650438Z",
     "start_time": "2024-05-09T07:15:06.644441Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-2.0, -2.0, -2.0, 10.0, 1.0, -1.0, 0.0]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_next_State_value(discount_factor,rewards,old_state_values,transition_probabilities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "340fda72",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:15:35.720463Z",
     "start_time": "2024-05-09T07:15:35.282162Z"
    }
   },
   "outputs": [],
   "source": [
    "for _ in range(100000):\n",
    "    new_state_values=calc_next_State_value(discount_factor,rewards,old_state_values,transition_probabilities)\n",
    "    old_state_values=new_state_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "ad945138",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T07:15:36.918567Z",
     "start_time": "2024-05-09T07:15:36.912580Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-12.543209876543184,\n",
       " 1.456790123456793,\n",
       " 4.320987654320991,\n",
       " 10.0,\n",
       " 0.8024691358024769,\n",
       " -22.543209876543163,\n",
       " 0.0]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_state_values"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
